# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
from urllib import parse


class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']

    # start_urls = ['http://blog.jobbole.com/114041/']

    def parse(self, response):
        # 解析列表页面,获取当前所有文章列表url并进行解析
        post_urls = response.css("#archive .floated-thumb .post-thumb a::attr(href)").extract()
        for post_url in post_urls:
            # response.url+post_url     不是完整url要ping域名
            yield Request(url=parse.urljoin(response.url, post_url), callback=self.parse_detail)
            # css选择器.class后不加空格表示并行,加空格表示包含
        next_url = response.css(".next.page-numbers::attr(href)").extract_first()
        if next_url:
            yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)

    def parse_detail(self, response):
        # 提取文章的具体字段
        title = response.xpath('//*[@id="post-114041"]/div[1]/h1/text()').extract_first()
